{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom JSON Encoding using JSONEncoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "In the previous video, we saw how we were able to provide custom encodings using the `default` argument of the `dump`/`dumps` function.\n",
    "\n",
    "But how does Python know how to encode the \"standard\" types, such as `str`, `int`, `float`, `list`, `dict`, etc?\n",
    "\n",
    "It uses a special class - `JSONEncoder`.\n",
    "\n",
    "This class supports the following encodings (see Python docs: https://docs.python.org/3/library/json.html#json.JSONEncoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|Python |JSON  |\n",
    "|:----|:---|\n",
    "| `dict` | object `{...}`|\n",
    "| `list`, `tuple` | array `[...]` |\n",
    "| `str`  | string `\"...\"`|\n",
    "| `int`, `float` | number |\n",
    "| `int` or `float` `Enums` | number |\n",
    "| `bool` | `true` or `false` |\n",
    "| `None` | `null` |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Anything beyond those Python types and we end up with a `TypeError` exception."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see how this class encodes objects by calling an instance of it directly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[1, 2, 3]'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "default_encoder = json.JSONEncoder()\n",
    "default_encoder.encode([1, 2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And for non-supported objects:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Object of type 'complex' is not JSON serializable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-faa062f02cec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdefault_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1j\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36mencode\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;31m# exceptions aren't as detailed.  The list call should be roughly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;31m# equivalent to the PySequence_Fast that ''.join() would do.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_one_shot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m             \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36miterencode\u001b[0;34m(self, o, _one_shot)\u001b[0m\n\u001b[1;32m    255\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m                 self.skipkeys, _one_shot)\n\u001b[0;32m--> 257\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36mdefault\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    178\u001b[0m         \"\"\"\n\u001b[1;32m    179\u001b[0m         raise TypeError(\"Object of type '%s' is not JSON serializable\" %\n\u001b[0;32m--> 180\u001b[0;31m                         o.__class__.__name__)\n\u001b[0m\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Object of type 'complex' is not JSON serializable"
     ]
    }
   ],
   "source": [
    "default_encoder.encode(1+1j)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can actually extend this `JSONEncoder` class and override the `default` method. We can then add in support for whatever type we want to use, and pass any other types to the parent class to handle (either serialize the data or raise a `TypeError` exception). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's just see a simple example first:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from datetime import datetime\n",
    "\n",
    "class CustomJSONEncoder(json.JSONEncoder):\n",
    "    def default(self, arg):\n",
    "        if isinstance(arg, datetime):\n",
    "            return arg.isoformat()\n",
    "        else:\n",
    "            super().default(arg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_encoder = CustomJSONEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'true'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_encoder.encode(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"2018-12-29T22:27:19.863377\"'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_encoder.encode(datetime.utcnow())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can now use this custom encoder by specifying it when we use `dump`/`dumps`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"name\": \"test\", \"time\": \"2018-12-29T22:27:20.135841\"}'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.dumps(dict(name='test', time=datetime.utcnow()), cls=CustomJSONEncoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One thing to note is that for both the `default` approach, and the `cls` approach, our method / encoder will only be used for types that Python cannot already serialize on its own (strings, integers, lists, etc)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_encoder(arg):\n",
    "    print('Custom encoder called...')\n",
    "    if isinstance(arg, str):\n",
    "        return f'some string: {arg}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we want to \"override\" `dumps` default encoding behavior for strings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"name\": \"Python\"}'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.dumps({'name': 'Python'}, default=custom_encoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, we cannot do that - because the argument is a \"recognized\" type (`str`), Python does not even call our `custom_encoder` function.\n",
    "\n",
    "And the same happens when we override the `default` method in our custom `JSONEncoder` class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at the signature for `dumps`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function dumps in module json:\n",
      "\n",
      "dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, sort_keys=False, **kw)\n",
      "    Serialize ``obj`` to a JSON formatted ``str``.\n",
      "    \n",
      "    If ``skipkeys`` is true then ``dict`` keys that are not basic types\n",
      "    (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped\n",
      "    instead of raising a ``TypeError``.\n",
      "    \n",
      "    If ``ensure_ascii`` is false, then the return value can contain non-ASCII\n",
      "    characters if they appear in strings contained in ``obj``. Otherwise, all\n",
      "    such characters are escaped in JSON strings.\n",
      "    \n",
      "    If ``check_circular`` is false, then the circular reference check\n",
      "    for container types will be skipped and a circular reference will\n",
      "    result in an ``OverflowError`` (or worse).\n",
      "    \n",
      "    If ``allow_nan`` is false, then it will be a ``ValueError`` to\n",
      "    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in\n",
      "    strict compliance of the JSON specification, instead of using the\n",
      "    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).\n",
      "    \n",
      "    If ``indent`` is a non-negative integer, then JSON array elements and\n",
      "    object members will be pretty-printed with that indent level. An indent\n",
      "    level of 0 will only insert newlines. ``None`` is the most compact\n",
      "    representation.\n",
      "    \n",
      "    If specified, ``separators`` should be an ``(item_separator, key_separator)``\n",
      "    tuple.  The default is ``(', ', ': ')`` if *indent* is ``None`` and\n",
      "    ``(',', ': ')`` otherwise.  To get the most compact JSON representation,\n",
      "    you should specify ``(',', ':')`` to eliminate whitespace.\n",
      "    \n",
      "    ``default(obj)`` is a function that should return a serializable version\n",
      "    of obj or raise TypeError. The default simply raises TypeError.\n",
      "    \n",
      "    If *sort_keys* is true (default: ``False``), then the output of\n",
      "    dictionaries will be sorted by key.\n",
      "    \n",
      "    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the\n",
      "    ``.default()`` method to serialize additional types), specify it with\n",
      "    the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(json.dumps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And let's see the signature for `JSONEncoder`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class JSONEncoder in module json.encoder:\n",
      "\n",
      "class JSONEncoder(builtins.object)\n",
      " |  Extensible JSON <http://json.org> encoder for Python data structures.\n",
      " |  \n",
      " |  Supports the following objects and types by default:\n",
      " |  \n",
      " |  +-------------------+---------------+\n",
      " |  | Python            | JSON          |\n",
      " |  +===================+===============+\n",
      " |  | dict              | object        |\n",
      " |  +-------------------+---------------+\n",
      " |  | list, tuple       | array         |\n",
      " |  +-------------------+---------------+\n",
      " |  | str               | string        |\n",
      " |  +-------------------+---------------+\n",
      " |  | int, float        | number        |\n",
      " |  +-------------------+---------------+\n",
      " |  | True              | true          |\n",
      " |  +-------------------+---------------+\n",
      " |  | False             | false         |\n",
      " |  +-------------------+---------------+\n",
      " |  | None              | null          |\n",
      " |  +-------------------+---------------+\n",
      " |  \n",
      " |  To extend this to recognize other objects, subclass and implement a\n",
      " |  ``.default()`` method with another method that returns a serializable\n",
      " |  object for ``o`` if possible, otherwise it should call the superclass\n",
      " |  implementation (to raise ``TypeError``).\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None)\n",
      " |      Constructor for JSONEncoder, with sensible defaults.\n",
      " |      \n",
      " |      If skipkeys is false, then it is a TypeError to attempt\n",
      " |      encoding of keys that are not str, int, float or None.  If\n",
      " |      skipkeys is True, such items are simply skipped.\n",
      " |      \n",
      " |      If ensure_ascii is true, the output is guaranteed to be str\n",
      " |      objects with all incoming non-ASCII characters escaped.  If\n",
      " |      ensure_ascii is false, the output can contain non-ASCII characters.\n",
      " |      \n",
      " |      If check_circular is true, then lists, dicts, and custom encoded\n",
      " |      objects will be checked for circular references during encoding to\n",
      " |      prevent an infinite recursion (which would cause an OverflowError).\n",
      " |      Otherwise, no such check takes place.\n",
      " |      \n",
      " |      If allow_nan is true, then NaN, Infinity, and -Infinity will be\n",
      " |      encoded as such.  This behavior is not JSON specification compliant,\n",
      " |      but is consistent with most JavaScript based encoders and decoders.\n",
      " |      Otherwise, it will be a ValueError to encode such floats.\n",
      " |      \n",
      " |      If sort_keys is true, then the output of dictionaries will be\n",
      " |      sorted by key; this is useful for regression tests to ensure\n",
      " |      that JSON serializations can be compared on a day-to-day basis.\n",
      " |      \n",
      " |      If indent is a non-negative integer, then JSON array\n",
      " |      elements and object members will be pretty-printed with that\n",
      " |      indent level.  An indent level of 0 will only insert newlines.\n",
      " |      None is the most compact representation.\n",
      " |      \n",
      " |      If specified, separators should be an (item_separator, key_separator)\n",
      " |      tuple.  The default is (', ', ': ') if *indent* is ``None`` and\n",
      " |      (',', ': ') otherwise.  To get the most compact JSON representation,\n",
      " |      you should specify (',', ':') to eliminate whitespace.\n",
      " |      \n",
      " |      If specified, default is a function that gets called for objects\n",
      " |      that can't otherwise be serialized.  It should return a JSON encodable\n",
      " |      version of the object or raise a ``TypeError``.\n",
      " |  \n",
      " |  default(self, o)\n",
      " |      Implement this method in a subclass such that it returns\n",
      " |      a serializable object for ``o``, or calls the base implementation\n",
      " |      (to raise a ``TypeError``).\n",
      " |      \n",
      " |      For example, to support arbitrary iterators, you could\n",
      " |      implement default like this::\n",
      " |      \n",
      " |          def default(self, o):\n",
      " |              try:\n",
      " |                  iterable = iter(o)\n",
      " |              except TypeError:\n",
      " |                  pass\n",
      " |              else:\n",
      " |                  return list(iterable)\n",
      " |              # Let the base class default method raise the TypeError\n",
      " |              return JSONEncoder.default(self, o)\n",
      " |  \n",
      " |  encode(self, o)\n",
      " |      Return a JSON string representation of a Python data structure.\n",
      " |      \n",
      " |      >>> from json.encoder import JSONEncoder\n",
      " |      >>> JSONEncoder().encode({\"foo\": [\"bar\", \"baz\"]})\n",
      " |      '{\"foo\": [\"bar\", \"baz\"]}'\n",
      " |  \n",
      " |  iterencode(self, o, _one_shot=False)\n",
      " |      Encode the given object and yield each string\n",
      " |      representation as available.\n",
      " |      \n",
      " |      For example::\n",
      " |      \n",
      " |          for chunk in JSONEncoder().iterencode(bigobject):\n",
      " |              mysocket.write(chunk)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes defined here:\n",
      " |  \n",
      " |  item_separator = ', '\n",
      " |  \n",
      " |  key_separator = ': '\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(json.JSONEncoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we are particularly interested in the `__init__` method signature:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " `__init__(self, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None)`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, sort_keys=False, **kw)`"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "You'll notice that there are quite a few arguments for both, most of which are common to both.\n",
    "\n",
    "With the `dump`/`dumps` method, there are quite a few things we can configure that control the json encoding - if we want to use those tweaks in a consistent manner throughout our app, we need to remember to use them consistently every time we call the `dump`/`dumps` function.\n",
    "\n",
    "Consider this example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {\n",
    "    'a': float('inf'),\n",
    "    'b': [1, 2, 3]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a': inf, 'b': [1, 2, 3]}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(d['a'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, that float is a special type of float - it represents + infinity."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see if Python can encode that:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"a\": Infinity, \"b\": [1, 2, 3]}'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.dumps(d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yes, it does - but notice the output, `Infinity`. Technically this is not JSON... (see https://tools.ietf.org/html/rfc4627 Section 2.4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, if we want to be strict about this, and ensure we are not trying to serialize a value such as infinity, we would do this instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Out of range float values are not JSON compliant",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-a7057d91995f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_nan\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/__init__.py\u001b[0m in \u001b[0;36mdumps\u001b[0;34m(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    236\u001b[0m         \u001b[0mcheck_circular\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheck_circular\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_nan\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mallow_nan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindent\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m         \u001b[0mseparators\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseparators\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdefault\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msort_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msort_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 238\u001b[0;31m         **kw).encode(obj)\n\u001b[0m\u001b[1;32m    239\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36mencode\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;31m# exceptions aren't as detailed.  The list call should be roughly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;31m# equivalent to the PySequence_Fast that ''.join() would do.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_one_shot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m             \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36miterencode\u001b[0;34m(self, o, _one_shot)\u001b[0m\n\u001b[1;32m    255\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m                 self.skipkeys, _one_shot)\n\u001b[0;32m--> 257\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,\n",
      "\u001b[0;31mValueError\u001b[0m: Out of range float values are not JSON compliant"
     ]
    }
   ],
   "source": [
    "json.dumps(d, allow_nan=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we get the desired result.\n",
    "\n",
    "What about trying to encode an invalid key (from JSON's perspective)::"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {10: \"int\", 10.5: \"float\", 1+1j: \"complex\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{10: 'int', 10.5: 'float', (1+1j): 'complex'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are all valid Python dictionary keys, but what happens with JSON encoding?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "keys must be a string",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-1a41a0b88650>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/__init__.py\u001b[0m in \u001b[0;36mdumps\u001b[0;34m(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    229\u001b[0m         \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mindent\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mseparators\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m         default is None and not sort_keys and not kw):\n\u001b[0;32m--> 231\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_default_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    232\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mJSONEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36mencode\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;31m# exceptions aren't as detailed.  The list call should be roughly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;31m# equivalent to the PySequence_Fast that ''.join() would do.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m         \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_one_shot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m             \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepdive/lib/python3.6/json/encoder.py\u001b[0m in \u001b[0;36miterencode\u001b[0;34m(self, o, _one_shot)\u001b[0m\n\u001b[1;32m    255\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem_separator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m                 self.skipkeys, _one_shot)\n\u001b[0;32m--> 257\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_iterencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    259\u001b[0m def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,\n",
      "\u001b[0;31mTypeError\u001b[0m: keys must be a string"
     ]
    }
   ],
   "source": [
    "json.dumps(d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see we get an exception. We may want to simply ignore that exception and not include the offending key/value pair in our serialization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"10\": \"int\", \"10.5\": \"float\"}'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.dumps(d, skipkeys=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now we no longer get an exception, and the complex key was simply skipped."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can even change how the serialization is rendered (which of course means we may no longer have actual JSON):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {\n",
    "    'name': 'Python',\n",
    "    'age': 27,\n",
    "    'created_by': 'Guido van Rossum',\n",
    "    'list': [1, 2, 3]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"name\": \"Python\", \"age\": 27, \"created_by\": \"Guido van Rossum\", \"list\": [1, 2, 3]}'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json.dumps(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "---\"name\" = \"Python\"\n",
      "---\"age\" = 27\n",
      "---\"created_by\" = \"Guido van Rossum\"\n",
      "---\"list\" = [\n",
      "------1\n",
      "------2\n",
      "------3\n",
      "---]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(d, indent='---', separators=('', ' = ')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can use this by the way, to create more compact JSON strings (uses less bytes):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"name\": \"Python\", \"age\": 27, \"created_by\": \"Guido van Rossum\", \"list\": [1, 2, 3]}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(d))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "vs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"name\":\"Python\",\"age\":27,\"created_by\":\"Guido van Rossum\",\"list\":[1,2,3]}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(d, separators=(',', ':')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, all the whitespace is eliminated. For transmitting large JSON objects, that can make a (relatively small) difference in making the JSON more compact."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, if we want to consistently use the same values for all those tweaks, we have to consistently remember to set the arguments correctly in the `dump`/`dumps` functions.\n",
    "\n",
    "Instead, we could create a custom JSONEncoder class that pre-sets all these things, and just use that encoder - simpler than remembering all those arguments and their correct values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomEncoder(json.JSONEncoder):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(skipkeys=True, \n",
    "                         allow_nan=False, \n",
    "                         indent='---', \n",
    "                         separators=('', ' = ')\n",
    "                        )\n",
    "        \n",
    "    def default(self, arg):\n",
    "        if isinstance(arg, datetime):\n",
    "            return arg.isoformat()\n",
    "        else:\n",
    "            return super().default(arg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {\n",
    "    'time': datetime.utcnow(),\n",
    "    1+1j: \"complex\",\n",
    "    'name': 'Python'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "---\"time\" = \"2018-12-29T22:27:26.689488\"\n",
      "---\"name\" = \"Python\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(d, cls=CustomEncoder))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another thing I want to point out is that with both these methods we are not limited in what we emit as our JSON serialization.\n",
    "\n",
    "For example, for a `datetime` object, we may want to emit not only the ISO formatted date, but maybe some additional fields, all nested within a JSON object:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomEncoder(json.JSONEncoder):\n",
    "    def default(self, arg):\n",
    "        if isinstance(arg, datetime):\n",
    "            obj = dict(\n",
    "                datatype=\"datetime\",\n",
    "                iso=arg.isoformat(),\n",
    "                date=arg.date().isoformat(),\n",
    "                time=arg.time().isoformat(),\n",
    "                year=arg.year,\n",
    "                month=arg.month,\n",
    "                day=arg.day,\n",
    "                hour=arg.hour,\n",
    "                minutes=arg.minute,\n",
    "                seconds=arg.second\n",
    "            )\n",
    "            return obj\n",
    "        else:\n",
    "            return super().default(arg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {\n",
    "    'time': datetime.utcnow(),\n",
    "    'message': 'Testing...'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"time\": {\n",
      "    \"datatype\": \"datetime\",\n",
      "    \"iso\": \"2018-12-29T22:27:27.668208\",\n",
      "    \"date\": \"2018-12-29\",\n",
      "    \"time\": \"22:27:27.668208\",\n",
      "    \"year\": 2018,\n",
      "    \"month\": 12,\n",
      "    \"day\": 29,\n",
      "    \"hour\": 22,\n",
      "    \"minutes\": 27,\n",
      "    \"seconds\": 27\n",
      "  },\n",
      "  \"message\": \"Testing...\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(d, cls=CustomEncoder, indent=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
